{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from awsglue.transforms import *\n",
    "from awsglue.utils import getResolvedOptions\n",
    "from pyspark.context import SparkContext\n",
    "from awsglue.context import GlueContext\n",
    "from awsglue.job import Job\n",
    "\n",
    "## @params: [JOB_NAME]\n",
    "args = getResolvedOptions(sys.argv, ['JOB_NAME'])\n",
    "\n",
    "sc = SparkContext()\n",
    "glueContext = GlueContext(sc)\n",
    "spark = glueContext.spark_session\n",
    "job = Job(glueContext)\n",
    "job.init(args['JOB_NAME'], args)\n",
    "## @type: DataSource\n",
    "## @args: [database = \"flights-db\", table_name = \"flightscsv\", transformation_ctx = \"datasource0\"]\n",
    "## @return: datasource0\n",
    "## @inputs: []\n",
    "datasource0 = glueContext.create_dynamic_frame.from_catalog(database = \"flights-db\", table_name = \"flightscsv\", transformation_ctx = \"datasource0\")\n",
    "## @type: ApplyMapping\n",
    "## @args: [mapping = [(\"year\", \"short\", \"year\", \"short\"), (\"quarter\", \"long\", \"quarter\", \"long\"), (\"month\", \"long\", \"month\", \"long\"), (\"day_of_month\", \"long\", \"day_of_month\", \"long\"), (\"day_of_week\", \"long\", \"day_of_week\", \"long\"), (\"fl_date\", \"string\", \"fl_date\", \"string\"), (\"unique_carrier\", \"string\", \"unique_carrier\", \"string\"), (\"airline_id\", \"long\", \"airline_id\", \"long\"), (\"carrier\", \"string\", \"carrier\", \"string\"), (\"tail_num\", \"string\", \"tail_num\", \"string\"), (\"fl_num\", \"long\", \"fl_num\", \"long\"), (\"origin_airport_id\", \"long\", \"origin_airport_id\", \"long\"), (\"origin_airport_seq_id\", \"long\", \"origin_airport_seq_id\", \"long\"), (\"origin_city_market_id\", \"long\", \"origin_city_market_id\", \"long\"), (\"origin\", \"string\", \"origin\", \"string\"), (\"origin_city_name\", \"string\", \"origin_city_name\", \"string\"), (\"origin_state_abr\", \"string\", \"origin_state_abr\", \"string\"), (\"origin_state_fips\", \"long\", \"origin_state_fips\", \"long\"), (\"origin_state_nm\", \"string\", \"origin_state_nm\", \"string\"), (\"origin_wac\", \"long\", \"origin_wac\", \"long\"), (\"dest_airport_id\", \"long\", \"dest_airport_id\", \"long\"), (\"dest_airport_seq_id\", \"long\", \"dest_airport_seq_id\", \"long\"), (\"dest_city_market_id\", \"long\", \"dest_city_market_id\", \"long\"), (\"dest\", \"string\", \"dest\", \"string\"), (\"dest_city_name\", \"string\", \"dest_city_name\", \"string\"), (\"dest_state_abr\", \"string\", \"dest_state_abr\", \"string\"), (\"dest_state_fips\", \"long\", \"dest_state_fips\", \"long\"), (\"dest_state_nm\", \"string\", \"dest_state_nm\", \"string\"), (\"dest_wac\", \"long\", \"dest_wac\", \"long\"), (\"crs_dep_time\", \"long\", \"crs_dep_time\", \"long\"), (\"dep_time\", \"long\", \"dep_time\", \"long\"), (\"dep_delay\", \"long\", \"dep_delay\", \"long\"), (\"dep_delay_new\", \"long\", \"dep_delay_new\", \"long\"), (\"dep_del15\", \"long\", \"dep_del15\", \"long\"), (\"dep_delay_group\", \"long\", \"dep_delay_group\", \"long\"), (\"dep_time_blk\", \"string\", \"dep_time_blk\", \"string\"), (\"taxi_out\", \"long\", \"taxi_out\", \"long\"), (\"wheels_off\", \"long\", \"wheels_off\", \"long\"), (\"wheels_on\", \"long\", \"wheels_on\", \"long\"), (\"taxi_in\", \"long\", \"taxi_in\", \"long\"), (\"crs_arr_time\", \"long\", \"crs_arr_time\", \"long\"), (\"arr_time\", \"long\", \"arr_time\", \"long\"), (\"arr_delay\", \"long\", \"arr_delay\", \"long\"), (\"arr_delay_new\", \"long\", \"arr_delay_new\", \"long\"), (\"arr_del15\", \"long\", \"arr_del15\", \"long\"), (\"arr_delay_group\", \"long\", \"arr_delay_group\", \"long\"), (\"arr_time_blk\", \"string\", \"arr_time_blk\", \"string\"), (\"cancelled\", \"long\", \"cancelled\", \"long\"), (\"cancellation_code\", \"string\", \"cancellation_code\", \"string\"), (\"diverted\", \"long\", \"diverted\", \"long\"), (\"crs_elapsed_time\", \"long\", \"crs_elapsed_time\", \"long\"), (\"actual_elapsed_time\", \"long\", \"actual_elapsed_time\", \"long\"), (\"air_time\", \"long\", \"air_time\", \"long\"), (\"flights\", \"long\", \"flights\", \"long\"), (\"distance\", \"long\", \"distance\", \"long\"), (\"distance_group\", \"long\", \"distance_group\", \"long\"), (\"carrier_delay\", \"long\", \"carrier_delay\", \"long\"), (\"weather_delay\", \"long\", \"weather_delay\", \"long\"), (\"nas_delay\", \"long\", \"nas_delay\", \"long\"), (\"security_delay\", \"long\", \"security_delay\", \"long\"), (\"late_aircraft_delay\", \"long\", \"late_aircraft_delay\", \"long\"), (\"first_dep_time\", \"long\", \"first_dep_time\", \"long\"), (\"total_add_gtime\", \"long\", \"total_add_gtime\", \"long\")], transformation_ctx = \"applymapping1\"]\n",
    "## @return: applymapping1\n",
    "## @inputs: [frame = datasource0]\n",
    "applymapping1 = ApplyMapping.apply(frame = datasource0, mappings = [(\"year\", \"short\", \"year\", \"short\"), (\"quarter\", \"long\", \"quarter\", \"long\"), (\"month\", \"long\", \"month\", \"long\"), (\"day_of_month\", \"long\", \"day_of_month\", \"long\"), (\"day_of_week\", \"long\", \"day_of_week\", \"long\"), (\"fl_date\", \"string\", \"fl_date\", \"string\"), (\"unique_carrier\", \"string\", \"unique_carrier\", \"string\"), (\"airline_id\", \"long\", \"airline_id\", \"long\"), (\"carrier\", \"string\", \"carrier\", \"string\"), (\"tail_num\", \"string\", \"tail_num\", \"string\"), (\"fl_num\", \"long\", \"fl_num\", \"long\"), (\"origin_airport_id\", \"long\", \"origin_airport_id\", \"long\"), (\"origin_airport_seq_id\", \"long\", \"origin_airport_seq_id\", \"long\"), (\"origin_city_market_id\", \"long\", \"origin_city_market_id\", \"long\"), (\"origin\", \"string\", \"origin\", \"string\"), (\"origin_city_name\", \"string\", \"origin_city_name\", \"string\"), (\"origin_state_abr\", \"string\", \"origin_state_abr\", \"string\"), (\"origin_state_fips\", \"long\", \"origin_state_fips\", \"long\"), (\"origin_state_nm\", \"string\", \"origin_state_nm\", \"string\"), (\"origin_wac\", \"long\", \"origin_wac\", \"long\"), (\"dest_airport_id\", \"long\", \"dest_airport_id\", \"long\"), (\"dest_airport_seq_id\", \"long\", \"dest_airport_seq_id\", \"long\"), (\"dest_city_market_id\", \"long\", \"dest_city_market_id\", \"long\"), (\"dest\", \"string\", \"dest\", \"string\"), (\"dest_city_name\", \"string\", \"dest_city_name\", \"string\"), (\"dest_state_abr\", \"string\", \"dest_state_abr\", \"string\"), (\"dest_state_fips\", \"long\", \"dest_state_fips\", \"long\"), (\"dest_state_nm\", \"string\", \"dest_state_nm\", \"string\"), (\"dest_wac\", \"long\", \"dest_wac\", \"long\"), (\"crs_dep_time\", \"long\", \"crs_dep_time\", \"long\"), (\"dep_time\", \"long\", \"dep_time\", \"long\"), (\"dep_delay\", \"long\", \"dep_delay\", \"long\"), (\"dep_delay_new\", \"long\", \"dep_delay_new\", \"long\"), (\"dep_del15\", \"long\", \"dep_del15\", \"long\"), (\"dep_delay_group\", \"long\", \"dep_delay_group\", \"long\"), (\"dep_time_blk\", \"string\", \"dep_time_blk\", \"string\"), (\"taxi_out\", \"long\", \"taxi_out\", \"long\"), (\"wheels_off\", \"long\", \"wheels_off\", \"long\"), (\"wheels_on\", \"long\", \"wheels_on\", \"long\"), (\"taxi_in\", \"long\", \"taxi_in\", \"long\"), (\"crs_arr_time\", \"long\", \"crs_arr_time\", \"long\"), (\"arr_time\", \"long\", \"arr_time\", \"long\"), (\"arr_delay\", \"long\", \"arr_delay\", \"long\"), (\"arr_delay_new\", \"long\", \"arr_delay_new\", \"long\"), (\"arr_del15\", \"long\", \"arr_del15\", \"long\"), (\"arr_delay_group\", \"long\", \"arr_delay_group\", \"long\"), (\"arr_time_blk\", \"string\", \"arr_time_blk\", \"string\"), (\"cancelled\", \"long\", \"cancelled\", \"long\"), (\"cancellation_code\", \"string\", \"cancellation_code\", \"string\"), (\"diverted\", \"long\", \"diverted\", \"long\"), (\"crs_elapsed_time\", \"long\", \"crs_elapsed_time\", \"long\"), (\"actual_elapsed_time\", \"long\", \"actual_elapsed_time\", \"long\"), (\"air_time\", \"long\", \"air_time\", \"long\"), (\"flights\", \"long\", \"flights\", \"long\"), (\"distance\", \"long\", \"distance\", \"long\"), (\"distance_group\", \"long\", \"distance_group\", \"long\"), (\"carrier_delay\", \"long\", \"carrier_delay\", \"long\"), (\"weather_delay\", \"long\", \"weather_delay\", \"long\"), (\"nas_delay\", \"long\", \"nas_delay\", \"long\"), (\"security_delay\", \"long\", \"security_delay\", \"long\"), (\"late_aircraft_delay\", \"long\", \"late_aircraft_delay\", \"long\"), (\"first_dep_time\", \"long\", \"first_dep_time\", \"long\"), (\"total_add_gtime\", \"long\", \"total_add_gtime\", \"long\")], transformation_ctx = \"applymapping1\")\n",
    "## @type: SelectFields\n",
    "## @args: [paths = [\"year\", \"quarter\", \"month\", \"day_of_month\", \"day_of_week\", \"fl_date\", \"unique_carrier\", \"airline_id\", \"carrier\", \"tail_num\", \"fl_num\", \"origin_airport_id\", \"origin_airport_seq_id\", \"origin_city_market_id\", \"origin\", \"origin_city_name\", \"origin_state_abr\", \"origin_state_fips\", \"origin_state_nm\", \"origin_wac\", \"dest_airport_id\", \"dest_airport_seq_id\", \"dest_city_market_id\", \"dest\", \"dest_city_name\", \"dest_state_abr\", \"dest_state_fips\", \"dest_state_nm\", \"dest_wac\", \"crs_dep_time\", \"dep_time\", \"dep_delay\", \"dep_delay_new\", \"dep_del15\", \"dep_delay_group\", \"dep_time_blk\", \"taxi_out\", \"wheels_off\", \"wheels_on\", \"taxi_in\", \"crs_arr_time\", \"arr_time\", \"arr_delay\", \"arr_delay_new\", \"arr_del15\", \"arr_delay_group\", \"arr_time_blk\", \"cancelled\", \"cancellation_code\", \"diverted\", \"crs_elapsed_time\", \"actual_elapsed_time\", \"air_time\", \"flights\", \"distance\", \"distance_group\", \"carrier_delay\", \"weather_delay\", \"nas_delay\", \"security_delay\", \"late_aircraft_delay\", \"first_dep_time\", \"total_add_gtime\", \"longest_add_gtime\"], transformation_ctx = \"selectfields2\"]\n",
    "## @return: selectfields2\n",
    "## @inputs: [frame = applymapping1]\n",
    "selectfields2 = SelectFields.apply(frame = applymapping1, paths = [\"year\", \"quarter\", \"month\", \"day_of_month\", \"day_of_week\", \"fl_date\", \"unique_carrier\", \"airline_id\", \"carrier\", \"tail_num\", \"fl_num\", \"origin_airport_id\", \"origin_airport_seq_id\", \"origin_city_market_id\", \"origin\", \"origin_city_name\", \"origin_state_abr\", \"origin_state_fips\", \"origin_state_nm\", \"origin_wac\", \"dest_airport_id\", \"dest_airport_seq_id\", \"dest_city_market_id\", \"dest\", \"dest_city_name\", \"dest_state_abr\", \"dest_state_fips\", \"dest_state_nm\", \"dest_wac\", \"crs_dep_time\", \"dep_time\", \"dep_delay\", \"dep_delay_new\", \"dep_del15\", \"dep_delay_group\", \"dep_time_blk\", \"taxi_out\", \"wheels_off\", \"wheels_on\", \"taxi_in\", \"crs_arr_time\", \"arr_time\", \"arr_delay\", \"arr_delay_new\", \"arr_del15\", \"arr_delay_group\", \"arr_time_blk\", \"cancelled\", \"cancellation_code\", \"diverted\", \"crs_elapsed_time\", \"actual_elapsed_time\", \"air_time\", \"flights\", \"distance\", \"distance_group\", \"carrier_delay\", \"weather_delay\", \"nas_delay\", \"security_delay\", \"late_aircraft_delay\", \"first_dep_time\", \"total_add_gtime\", \"longest_add_gtime\"], transformation_ctx = \"selectfields2\")\n",
    "## @type: ResolveChoice\n",
    "## @args: [choice = \"MATCH_CATALOG\", database = \"flights-db\", table_name = \"flightscsv\", transformation_ctx = \"resolvechoice3\"]\n",
    "## @return: resolvechoice3\n",
    "## @inputs: [frame = selectfields2]\n",
    "resolvechoice3 = ResolveChoice.apply(frame = selectfields2, choice = \"MATCH_CATALOG\", database = \"flights-db\", table_name = \"flightscsv\", transformation_ctx = \"resolvechoice3\")\n",
    "## @type: DataSink\n",
    "## @args: [database = \"flights-db\", table_name = \"flightscsv\", transformation_ctx = \"datasink4\"]\n",
    "## @return: datasink4\n",
    "## @inputs: [frame = resolvechoice3]\n",
    "datasink4 = glueContext.write_dynamic_frame.from_catalog(frame = resolvechoice3, database = \"flights-db\", table_name = \"flightscsv\", transformation_ctx = \"datasink4\")\n",
    "job.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
